# -*- coding: utf-8 -*-
import scrapy
import json

from scrapy import Spider, Request,FormRequest
from MultipleSpider.items import ProxyItem

class ProxySpider(scrapy.Spider):
    name = 'proxy'
    allowed_domains = ['kuaidaili.com']
    page_url = 'https://www.kuaidaili.com/ops/proxylist/{pageNum}/'
    download_timeout = 10

    def start_requests(self):
        for index in range(10):
            yield Request(self.page_url.format(pageNum=index + 1),self.parse,dont_filter=True)

    def parse(self, response):
        list = response.css('#freelist tr td ::text').extract()

        count = int ( len(list) / 8 );
        for index in range(count):
            item = ProxyItem()
            item['ip'] = list[index * 8 + 0].lower().replace(' ', '')
            item['port'] = list[index * 8 + 1].lower().replace(' ', '')
            item['user'] = list[index * 8 + 2].lower().replace(' ', '')
            item['protocol'] = list[index * 8 + 3].lower().replace(' ', '')
            item['method'] = list[index * 8 + 4].lower().replace(' ', '')
            item['region'] = list[index * 8 + 5].lower()
            item['speed'] = list[index * 8 + 6].lower().replace(' ', '')

            proxy_http_url = 'http://%s:%s' % (item['ip'],item['port'])
            proxy_https_url = 'https://%s:%s' % (item['ip'],item['port'])
            req_url = 'http://httpbin.org/ip' #'http://www.baidu.com'

            #验证代理
            if 'https' in item['method']:
                yield Request(req_url,
                              meta={'key':item,'proxy':proxy_https_url},
                              callback=self.proxy_test,
                              dont_filter=True)
            else:
                yield Request(req_url,
                              meta={'key':item,'proxy':proxy_http_url},
                              callback=self.proxy_test,
                              dont_filter=True)
        pass

    def proxy_test(self, response):

        #
        # if response.status == 200:
        #     item = response.meta['key']
        #     yield item

        results = json.loads(response.text)
        if 'origin' in results.keys():
            ip = results.get('origin').lower().replace(' ', '')
            item = response.meta['key']
            if ip == item['ip']:
                yield item
        pass



#response.css('#freelist tr td ::text').extract()
#['222.188.188.31', '6666', '', 'HTTP, HTTPS', 'GET, POST', '   ', '3', '4', '115.223.224.250', '9000', '', 'HTTP', 'GET, POST', '  ', '2', '7', '115.223.254.242', '9000', '', 'HTTP', 'GET, POST', '  ', '3', '10', '115.216.38.174', '9000', '', 'HTTP', 'GET, POST', '  ', '2', '13', '101.71.233.145', '9000', '', 'HTTP', 'GET, POST', '   ', '0.6', '16', '101.71.239.106', '9000', '', 'HTTP', 'GET, POST', '   ', '2', '19', '115.223.254.28', '9000', '', 'HTTP', 'GET, POST', '  ', '3', '22', '115.223.211.49', '9000', '', 'HTTP', 'GET, POST', '  ', '3', '25', '101.71.239.211', '9000', '', 'HTTP', 'GET, POST', '  ', '3', '28', '115.223.241.238', '9000', '', 'HTTP', 'GET, POST', '  ', '0.3', '31']

